﻿using Aspose.Words;
using AuthorityCenter.Model;
using CsvHelper;
using ImgModelManage.Model;
using LdtCommonFreeSql;
using LdtCommonLibraryHttp;
using LdtCommonUtil;
using LdtCommonUtil.Security;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
using TxtSearch.Model;
using static FreeSql.Internal.GlobalFilter;
using static System.Net.Mime.MediaTypeNames;

namespace TxtSearch.Bll
{
    public class ServiceBll
    {
        private readonly ILogger<ServiceBll> logger;
        private readonly IHttpHelper httpHelper;
        private readonly IFreeSql freesqlCaseCauseCreate;
        private readonly string modelPath;
        public ServiceBll(ILogger<ServiceBll> logger, FreesqlDb freesqlDb, IHttpHelper httpHelper, IConfiguration configuration)
        {
            this.logger = logger;
            this.httpHelper = httpHelper;
            freesqlCaseCauseCreate = freesqlDb.GetFreeSql("CaseCauseCreate", new ConnConfig
            {
                Dbcode = "casecausecreate",
                Dbtype = FreeSql.DataType.PostgreSQL,
                IP = "192.168.1.52",
                MaxPool = "1",
                Password = "ldtlaw@123",
                Username = "ldt"
            });
            modelPath = configuration["ModelPath"];
        }
        /// <summary>
        /// Creates the datas by path.将文件路径下的数据导入到数据库中
        /// </summary>
        /// <param name="path">The path.</param>
        /// <param name="isMangSign">if set to <c>true</c> [is mang sign].</param>
        /// <param name="modelCode">The model code.</param>
        /// <returns></returns>
        public async Task CreateDatasByPath(string path, bool isMangSign, string modelCode)
        {
            var ds = new List<Classifydatas>();
            var files = Directory.GetFileSystemEntries(path, "*", SearchOption.AllDirectories);
            files.Where(p => p.EndsWith("txt", System.StringComparison.OrdinalIgnoreCase)
                || p.EndsWith("docx", System.StringComparison.OrdinalIgnoreCase)
                || p.EndsWith("doc", System.StringComparison.OrdinalIgnoreCase)).ToList().ForEach(f =>
            {
                string c1;
                if (f.EndsWith("txt", System.StringComparison.OrdinalIgnoreCase))
                {
                    c1 = File.ReadAllText(f);
                }
                else
                {
                    //word文件
                    using (FileStream fileStream = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    {
                        c1 = GetDocContent(new Document(fileStream));
                    }
                }
                ds.Add(new Classifydatas
                {
                    Content = c1,
                    Type = new FileInfo(f).Directory.Name,
                    ModelCode = modelCode,
                    Md5 = c1.MDString()
                });
            });
            if (isMangSign)
            {
                var temp1 = ds.GroupBy(p => p.Content);
                var ds2 = new List<Classifydatas>();
                foreach (var item in temp1)
                {
                    ds2.Add(new Classifydatas
                    {
                        Content = item.Key,
                        Type = string.Join(",", item.OrderBy(p => p.Type).Select(p => p.Type).Distinct()),
                        ModelCode = modelCode,
                        Md5 = item.First().Md5
                    });
                }
                ds = ds2;
            }
            await freesqlCaseCauseCreate.Insert(ds).ExecuteAffrowsAsync();
        }

        /// <summary>
        /// Creates the datas by path.将文件路径下的数据导入到数据库中
        /// </summary>
        /// <param name="path">The path.</param>
        /// <param name="isMangSign">if set to <c>true</c> [is mang sign].</param>
        /// <param name="modelCode">The model code.</param>
        /// <returns></returns>
        public async Task CreateDatasByModelTable(string modelCode)
        {
            var datas = freesqlCaseCauseCreate.Ado.Query<Classifydatas>($"select * from {modelCode.ToLower()}").ToList();
            datas = datas.Select(p => new Classifydatas
            {
                Content = p.Content,
                Type = p.Type,
                ModelCode = modelCode,
                Md5 = p.Content.MDString()
            }).ToList();
            await freesqlCaseCauseCreate.Delete<Classifydatas>().Where(p => p.ModelCode == modelCode).ExecuteAffrowsAsync();
            await freesqlCaseCauseCreate.Insert(datas).ExecuteAffrowsAsync();
            await CreateCsv(modelCode);
        }
        /// <summary>
        /// Creates the datas by path.将文件路径下的数据导入到数据库中
        /// </summary>
        /// <param name="path">The path.</param>
        /// <param name="isMangSign">if set to <c>true</c> [is mang sign].</param>
        /// <param name="modelCode">The model code.</param>
        /// <returns></returns>
        public async Task CreateDatasByModel(string modelCode)
        {
            var model = JsonConvert.DeserializeObject<ModelConfig>(File.ReadAllText(Path.Combine(modelPath, "App", "modelconfig", modelCode)));

            var ds = new List<Classifydatas>();
            foreach (var item in model.ModelSignConfigs)
            {
                var files = Directory.GetFileSystemEntries(Path.Combine(modelPath, "CaseCauseDatas", modelCode, item.SignName), "*", SearchOption.AllDirectories);
                files.Where(p => p.EndsWith("txt", System.StringComparison.OrdinalIgnoreCase)
                    || p.EndsWith("docx", System.StringComparison.OrdinalIgnoreCase)
                    || p.EndsWith("doc", System.StringComparison.OrdinalIgnoreCase)).ToList().ForEach(f =>
                    {
                        string c1;
                        if (f.EndsWith("txt", System.StringComparison.OrdinalIgnoreCase))
                        {
                            c1 = File.ReadAllText(f);
                        }
                        else
                        {
                            //word文件
                            using (FileStream fileStream = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                            {
                                c1 = GetDocContent(new Document(fileStream));
                            }
                        }
                        ds.Add(new Classifydatas
                        {
                            Content = c1,
                            Type = item.SignCode,
                            ModelCode = modelCode,
                            Md5 = c1.MDString()
                        });
                    });
            }
            var temp1 = ds.GroupBy(p => p.Content);
            var ds2 = new List<Classifydatas>();
            foreach (var item in temp1)
            {
                ds2.Add(new Classifydatas
                {
                    Content = item.Key,
                    Type = string.Join(",", item.OrderBy(p => p.Type).Select(p => p.Type).Distinct()),
                    ModelCode = modelCode,
                    Md5 = item.First().Md5
                });
            }
            ds = ds2;
            await freesqlCaseCauseCreate.Delete<Classifydatas>().Where(p => p.ModelCode == modelCode).ExecuteAffrowsAsync();
            await freesqlCaseCauseCreate.Insert(ds).ExecuteAffrowsAsync();
            await CreateCsv(modelCode);
        }
        /// <summary>
        /// Creates the CSV.生成向量回写 并生成csv文件
        /// </summary>
        public async Task CreateCsv(string modelCode)
        {
            var noMd5datas = freesqlCaseCauseCreate.Select<Classifydatas>()
    .Where(p => p.ModelCode == modelCode && string.IsNullOrEmpty(p.Md5)).ToList();
            noMd5datas = noMd5datas.Where(p => !string.IsNullOrWhiteSpace(p.Content)).Select(p => new Classifydatas
            {
                Id = p.Id,
                Content = p.Content,
                Type = p.Type,
                ModelCode = modelCode,
                State = p.State,
                Md5 = p.Content.MDString(),
            }).ToList();
            if (noMd5datas.Any())
            {
                freesqlCaseCauseCreate.InsertOrUpdate<Classifydatas>().SetSource(noMd5datas).ExecuteAffrows();
            }
            var datas = freesqlCaseCauseCreate.Select<Classifydatas>()
                .Where(p => p.ModelCode == modelCode &&
                    !freesqlCaseCauseCreate.Select<Embedding>().Any(a => a.Md5 == p.Md5))
                .ToList();
            var i = 0;
            while (true)
            {
                var iis = datas.Skip(i).Take(5).ToList();
                if (!iis.Any())
                {
                    break;
                }
                var tasks = new List<Task>();
                foreach (var data in iis)
                {
                    tasks.Add(CreateEm(data, modelCode));
                }
                Task.WaitAll(tasks.ToArray());
                i = i + 5;
            }
            //生成csv文件
            var cc1 = freesqlCaseCauseCreate.Select<Classifydatas, Embedding>()
                .Where((a, b) => a.ModelCode == modelCode)
                .Where((a, b) => a.Md5 == b.Md5)
                .ToList((a, b) => new
                {
                    Id = a.Id,
                    Embedding = b.Val,
                    Type = a.Type,
                });

            var alldatas = cc1.Where(p => !string.IsNullOrEmpty(p.Embedding)).GroupBy(p => p.Type);
            using (var writer = new StreamWriter("train.csv"))
            {
                using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture);
                Random rrr = new Random();
                foreach (var item in alldatas)
                {
                    var arrs = item.Select(p => new
                    {
                        id = p.Id,
                        embedding = p.Embedding
                    }).OrderBy(p => rrr.Next()).ToArray();
                    csv.WriteRecords(arrs);
                }
            }
            var r1 = httpHelper.Get($"http://127.0.0.1:8000/qa/adddata?filename=train.csv&table_name={modelCode}", null, 600000);
            logger.LogInformation($"重新生成完成：{r1}");
            //await ValidData(modelCode);
        }
        private async Task CreateEm(Classifydatas d, string modelCode)
        {
            try
            {
                var mess = await httpHelper.PostAsync($"http://127.0.0.1:8000/qa/encode", JsonConvert.SerializeObject(new
                {
                    content = d.Content.Length > 2048 ? d.Content.Substring(0, 2048) : d.Content,
                    tableName = modelCode
                }), 100000);
                if (mess.IsSuccessStatusCode)
                {
                    var str = await mess.Content.ReadAsStringAsync();

                    var obj = JsonConvert.DeserializeObject<HttpReturnEntity>(str);
                    if (obj.Code == "1")
                    {
                        await freesqlCaseCauseCreate.Insert(new Embedding
                        {
                            Md5 = d.Md5,
                            Val = obj.Message
                        }).ExecuteAffrowsAsync();
                    }
                }
            }
            catch (Exception ex)
            {
                logger.LogError(ex.Message);
            }
        }
        /// <summary>
        /// Searches the datas.获取搜索结果
        /// </summary>
        /// <param name="content">The content.</param>
        /// <param name="tableName">Name of the table.</param>
        /// <returns></returns>
        /// <exception cref="LdtCommonUtil.ServiceException">检索异常</exception>
        public async Task<List<List<string>>> SearchDatas(string content, string tableName, string machineresult = null)
        {
            var mess = await httpHelper.PostAsync($"http://127.0.0.1:8000/qa/search", JsonConvert.SerializeObject(new
            {
                content = content.Length > 2048 ? content.Substring(0, 2048) : content,
                embedding = machineresult,
                tableName = tableName
            }), 100000);
            if (mess.IsSuccessStatusCode)
            {
                var str = await mess.Content.ReadAsStringAsync();

                var obj = JsonConvert.DeserializeObject<HttpReturnEntity>(str);
                if (obj.Code == "1")
                {
                    var r1 = (obj.Body as JArray).ToObject<List<List<string>>>();
                    return r1;
                }
            }
            throw new ServiceException("检索异常");
        }
        /// <summary>
        /// Searches the datas.获取搜索结果
        /// </summary>
        /// <param name="content">The content.</param>
        /// <param name="tableName">Name of the table.</param>
        /// <returns></returns>
        /// <exception cref="LdtCommonUtil.ServiceException">检索异常</exception>
        public async Task<object> GetSearchResults(List<string> contents, string tableName, string machineresult = null)
        {
            List<List<SignResult>> results = new List<List<SignResult>>();
            foreach (var item in contents)
            {
                var r1 = await GetSearchResult(item, tableName, machineresult);
                results.Add(r1);
            }
            return new
            {
                result = new HttpReturnEntity
                {
                    Code = "1",
                    Message = "success"
                },
                body = new
                {
                    @class = results
                }
            };
        }
        /// <summary>
        /// Searches the datas.获取搜索结果
        /// </summary>
        /// <param name="content">The content.</param>
        /// <param name="tableName">Name of the table.</param>
        /// <returns></returns>
        /// <exception cref="LdtCommonUtil.ServiceException">检索异常</exception>
        private async Task<List<SignResult>> GetSearchResult(string content, string tableName, string machineresult = null)
        {
            var r1 = await SearchDatas(content, tableName, machineresult);
            List<SignResult> strs = new List<SignResult>();
            var labels = freesqlCaseCauseCreate.Select<Classifydatas>().Where(p => p.ModelCode == "labeldqqingxing").ToList();
            for (int i = r1[1].Count - 1; i >= 0; i--)
            {
                if (Convert.ToDouble(r1[1][i]) > 0.3)
                {
                    var labID = r1[0][i];
                    var lab = labels.Find(p => p.Id.ToString() == labID).Type;
                    strs.Add(new SignResult
                    {
                        name = lab,
                        probnum = r1[1][i]
                    });
                }
            }
            return strs;
        }
        public async Task CreateResults(string modelCode, string tableName)
        {
            var datas = freesqlCaseCauseCreate.Select<Classifydatas, Embedding>().Where(
    (a, b) => a.ModelCode == modelCode && a.Md5 == b.Md5)
    .ToList((a, b) => new Classifydatas
    {
        Id = a.Id,
        Content = a.Content,
        Machineresult = b.Val
    });
            for (int i = 0; i < datas.Count; i = i + 5)
            {
                List<Task> tasks = new List<Task>();
                for (int j = i; j < i + 5 && j < datas.Count; j++)
                {
                    var item = datas[j];
                    var task1 = SearchDatas(item.Content, tableName, item.Machineresult).ContinueWith(async rr1 =>
                    {
                        var r1 = await rr1;
                        var ins = r1[0].Select(a => int.Parse(a)).ToList();
                        var rtemp1 = await freesqlCaseCauseCreate.Select<Classifydatas>().Where(p => ins.Contains(p.Id))
                        .ToListAsync(p => new Classifydatas
                        {
                            Content = p.Content,
                            Id = p.Id
                        });
                        var str = ins.Select(p => rtemp1.Find(a => a.Id == p).Content).ToList();
                        await freesqlCaseCauseCreate.InsertOrUpdate<SimilarityResult>().SetSource(new SimilarityResult
                        {
                            SourceContent = item.Content,
                            SimResults = string.Join("\r\n-----------------", str)
                        }).ExecuteAffrowsAsync();
                    });
                    tasks.Add(task1);
                }
                Task.WaitAll(tasks.ToArray());
            }
            await Task.CompletedTask;
            logger.LogInformation("校验完成");
        }
        /// <summary>
        /// Valids the data.
        /// </summary>
        /// <param name="tableName">Name of the table.</param>
        public async Task ValidData(string tableName)
        {
            var datas = freesqlCaseCauseCreate.Select<Classifydatas, Embedding>().Where(
                (a, b) => a.ModelCode == tableName && a.Md5 == b.Md5 && a.State == "valid")
                .ToList((a, b) => new Classifydatas
                {
                    Id = a.Id,
                    Content = a.Content,
                    Machineresult = b.Val
                });
            for (int i = 0; i < datas.Count; i = i + 5)
            {
                List<Task> tasks = new List<Task>();
                for (int j = i; j < i + 5 && j < datas.Count; j++)
                {
                    var item = datas[j];
                    var task1 = SearchDatas(item.Content, tableName, item.Machineresult).ContinueWith(async rr1 =>
                    {
                        var r1 = await rr1;
                        await freesqlCaseCauseCreate.Update<Classifydatas>()
                             .Where(p => p.Id == item.Id)
                             .Set(p => p.Log, JsonConvert.SerializeObject(r1))
                             .Set(p => p.Machineresult, freesqlCaseCauseCreate.Select<Classifydatas>()
                                                                 .Where(a => a.Id == int.Parse(r1[0][0]))
                                                                 .First().Type
                                                                 )
                             .Set(p => p.Firstid, int.Parse(r1[0][0]))
                             .Set(p => p.Twoid, int.Parse(r1[0][1]))
                             .ExecuteAffrowsAsync();
                    });
                    tasks.Add(task1);
                }
                Task.WaitAll(tasks.ToArray());
            }
            await Task.CompletedTask;
            logger.LogInformation("校验完成");
        }
        /// <summary>
        /// Valids the data.
        /// </summary>
        /// <param name="tableName">Name of the table.</param>
        public async Task CreateTrainData(string tableName)
        {
            var clsData = new List<TrainClsData>();
            var trainFile = Path.Combine(AppContext.BaseDirectory, $"snli_zh_50.jsonl");
            if (File.Exists(trainFile))
            {
                var t1 = File.ReadAllLines(trainFile);
                clsData = t1.Select(p => JsonConvert.DeserializeObject<TrainClsData>(p)).ToList();
            }
            var datas = freesqlCaseCauseCreate.Select<Classifydatas>().Where(
                (a) => a.ModelCode == tableName)
                .ToList();
            datas = datas.Where(p => clsData.Exists(a => a.text1 == p.Content || a.text2 == p.Content)).ToList();
            await freesqlCaseCauseCreate.Ado.QueryAsync<dynamic>($"DELETE from embedding where md5 in (SELECT md5 from classifydatas where modelCode='{tableName}')");

            var errorDatas = await freesqlCaseCauseCreate.Ado.QueryAsync<dynamic>($"""
select a.id,a.firstid,a.twoid, a.content as content1,b.content as content2,a.type as type1,b.type as type2 FROM classifydatas a INNER JOIN (select id,content,type from classifydatas) b on a.firstid=b.id  where modelCode='{tableName}' and state='valid' and a.type!=b.type
union  
select a.id,a.firstid,a.twoid, a.content as content1, b.content as content2, a.type as type1, b.type as type2 FROM classifydatas a INNER JOIN (select id, content, type from classifydatas) b on a.twoid=b.id where modelCode='{tableName}' and state = 'train' and a.type!=b.type;
""");
            foreach (var item in errorDatas)
            {
                clsData.Add(new TrainClsData
                {
                    text1 = item["content1"].ToString(),
                    text2 = item["content2"].ToString(),
                    label = 0,
                });
                var t1 = datas.Where(p => p.Content != item["content1"].ToString() && p.Type == item["type1"].ToString())
                    .OrderBy(p => p.Id).FirstOrDefault();
                if (t1 != null)
                {
                    clsData.Add(new TrainClsData
                    {
                        text1 = item["content1"].ToString(),
                        text2 = t1.Content,
                        label = 1,
                    });
                    datas.Remove(t1);
                }
            }
            clsData = clsData.Distinct(p => $"{p.text1}{p.label}{p.text2}").ToList();
            File.WriteAllLines(trainFile, clsData.Select(p => JsonConvert.SerializeObject(p)));
            logger.LogInformation("生成校验数据完成");
        }
        private string GetDocContent(Document doc)
        {
            try
            {
                string result2 = "";
                NodeCollection text = doc.GetChildNodes(NodeType.Paragraph, isDeep: true);
                Dictionary<int, int> dic = new Dictionary<int, int>();
                for (int i = 0; i < text.Count; i++)
                {
                    Paragraph p = text[i] as Paragraph;
                    if (p.IsListItem)
                    {
                        int current = p.ListFormat.List.ListId;
                        if (!dic.ContainsKey(current))
                        {
                            dic.Add(current, 1);
                        }
                        else
                        {
                            dic[current]++;
                        }
                        string t3 = p.GetText();
                        t3 = p.ListFormat.List.ListLevels[0].NumberFormat.Replace("\0", dic[current].ToString()) + t3;
                        result2 += t3;
                    }
                    else
                    {
                        string t = p.GetText();
                        result2 += t;
                    }
                }
                return result2;
            }
            catch
            {
                return "";
            }
        }


        /// <summary>
        /// Creates the datas by path.将文件路径下的数据导入到数据库中
        /// </summary>
        /// <param name="path">The path.</param>
        /// <param name="isMangSign">if set to <c>true</c> [is mang sign].</param>
        /// <param name="modelCode">The model code.</param>
        /// <returns></returns>
        public async Task ConvertPathDataByModel(string modelCode)
        {
            var model = JsonConvert.DeserializeObject<ModelConfig>(File.ReadAllText(Path.Combine(modelPath, "App", "modelconfig", modelCode)));

            var ds = new List<CommonClassifyDatas>();
            var ccids = new List<CcidReleation>();
            foreach (var item in model.ModelSignConfigs)
            {
                if (Directory.Exists(Path.Combine(modelPath, "CaseCauseDatas", modelCode, item.SignName)))
                {
                    var files = Directory.GetFileSystemEntries(Path.Combine(modelPath, "CaseCauseDatas", modelCode, item.SignName), "*", SearchOption.AllDirectories);
                    files.Where(p => p.EndsWith("txt", System.StringComparison.OrdinalIgnoreCase)
                        || p.EndsWith("docx", System.StringComparison.OrdinalIgnoreCase)
                        || p.EndsWith("doc", System.StringComparison.OrdinalIgnoreCase)).ToList().ForEach(f =>
                        {
                            string c1;
                            if (f.EndsWith("txt", System.StringComparison.OrdinalIgnoreCase))
                            {
                                c1 = File.ReadAllText(f);
                            }
                            else
                            {
                                //word文件
                                using (FileStream fileStream = new FileStream(f, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                                {
                                    c1 = GetDocContent(new Document(fileStream));
                                }
                            }
                            ds.Add(new CommonClassifyDatas
                            {
                                Content = c1,
                                Type = item.SignCode,
                                Id = c1.MDString()
                            });
                            //var code1 = item.SignCode.Split("_");
                            //ccids.Add(new CcidReleation
                            //{
                            //    Ccid = code1[0],
                            //    Ccname = code1.Length > 1 ? code1[1] : item.SignName
                            //});
                            //ccids.Add(new CcidReleation
                            //{
                            //    Ccid = item.SignCode,
                            //    Ccname = item.SignName
                            //});
                        });
                }
            }
            ccids = ccids.Distinct(p => p.Ccid).ToList();
            var ttttt1 = ds.GroupBy(p => p.Id).Where(p => p.Select(a => a.Type).Distinct().Count() > 1).ToList();
            var temp1 = ds.Distinct(p => p.Id).ToList();
            await freesqlCaseCauseCreate.Insert(temp1).AsTable((a) => modelCode.ToLower()).ExecuteAffrowsAsync();
            await freesqlCaseCauseCreate.InsertOrUpdate<CcidReleation>().SetSource(ccids).IfExistsDoNothing().ExecuteAffrowsAsync();
        }

        /// <summary>
        /// Valids the data.
        /// </summary>
        /// <param name="tableName">Name of the table.</param>
        public async Task CreateTrainData2(string tableName)
        {
            Random random = new Random();
            var clsData = new List<TrainClsData>();
            var trainFile = Path.Combine(AppContext.BaseDirectory, $"snli_zh_50.jsonl");
            var datas = await freesqlCaseCauseCreate.Ado.QueryAsync<dynamic>($"select * from {tableName} where istrain=1");
            var labels = datas.SelectMany(p => $"{p["type"]}".Split(",", StringSplitOptions.RemoveEmptyEntries));
            labels = labels.Select(p => p.Split("_")[1]).Distinct().ToList();
            foreach (var item in datas)
            {
                var ttt1 = new TrainClsData
                {
                    text1 = item["content"].ToString(),
                    text2 = item["type"].ToString(),
                    label = 1,
                };
                var ttt0 = ttt1.text2.Split(",", StringSplitOptions.RemoveEmptyEntries).ToList();
                clsData.AddRange(ttt0.Select(p => new TrainClsData
                {
                    text1 = item["content"].ToString(),
                    label = 1,
                    text2 = p.Split("_")[1],
                }));
                clsData.AddRange(labels.Where(a => !ttt0.Exists(bb => bb.Contains(a))).Select(p => new TrainClsData
                {
                    text1 = item["content"].ToString(),
                    label = 0,
                    text2 = p,
                }));
            }
            clsData = clsData.Distinct(p => $"{p.text1}{p.label}{p.text2}").OrderBy(a => random.Next()).ToList();
            File.WriteAllLines(trainFile, clsData.Select(p => JsonConvert.SerializeObject(p)));
            logger.LogInformation("生成校验数据完成");
        }
    }
}
